{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-\n",
    "# @Time    : 2017/11/13 10:55\n",
    "# @Author  : Deyu.Tian\n",
    "# @Site    : \n",
    "# @File    : dataClean.py\n",
    "# @Software: PyCharm Community Edition\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import config\n",
    "tempDir = '{}\\\\temperture'.format(config.obsvDataDir)\n",
    "precDir = '{}\\\\preciption'.format(config.obsvDataDir)\n",
    "evapDir = '{}\\\\evaporation'.format(config.obsvDataDir)\n",
    "surfDir = '{}\\\\surfaceTemp'.format(config.obsvDataDir)\n",
    "windDir = '{}\\\\windSpeed'.format(config.obsvDataDir)\n",
    "\n",
    "from util import *\n",
    "TEMPS = list_all_texts(tempDir)\n",
    "PRECS = list_all_texts(precDir)\n",
    "EVAPS = list_all_texts(evapDir)\n",
    "SURFS = list_all_texts(surfDir)\n",
    "WINDS = list_all_texts(windDir)\n",
    "ATMOS = [TEMPS, PRECS]\n",
    "\n",
    "\n",
    "\n",
    "def _post_processing(atmos):\n",
    "    \"\"\"\n",
    "    对合并后表格后处理\n",
    "    :return: 处理后的表格\n",
    "    \"\"\"\n",
    "    atmos.columns = ['year_month_day', 'stationID', 'maxDayTemp', 'minDayTemp', '_year_month_day', '_stationID',\n",
    "                     'accumPrec']\n",
    "    atmos['accumPrec_1'] = atmos['accumPrec']\n",
    "\n",
    "    for index, row in atmos.iterrows():\n",
    "        if row['accumPrec_1'] == 32700:\n",
    "            atmos.loc[index, 'accumPrec_1'] = 0\n",
    "        elif row['accumPrec_1'] >= 32000 and row['accumPrec_1'] < 32700:\n",
    "            atmos.loc[index, 'accumPrec_1'] = row['accumPrec_1'] - 32000\n",
    "        elif row['accumPrec_1'] >= 31000 and row['accumPrec_1'] < 32000:\n",
    "            atmos.loc[index, 'accumPrec_1'] = row['accumPrec_1'] - 31000\n",
    "        elif row['accumPrec_1'] >= 30000 and row['accumPrec_1'] < 31000:\n",
    "            atmos.loc[index, 'accumPrec_1'] = row['accumPrec_1'] - 30000\n",
    "    return atmos\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def combine_all_files():\n",
    "    temp = pd.read_table(TEMPS[0], sep='\\s+', usecols=[0, 4, 5, 6, 8, 9],\n",
    "                         names=['stationID', 'year', 'month', 'day', 'maxDayTemp', 'minDayTemp'],\n",
    "                         dtype={'stationID': np.string0, 'maxDayTemp': np.float32, 'minDayTemp': np.float32},\n",
    "                         parse_dates=[[1, 2, 3]])\n",
    "    prec = pd.read_table(PRECS[0], sep='\\s+', usecols=[0, 4, 5, 6, 9],\n",
    "                         names=['stationID', 'year', 'month', 'day', 'accumPrec'],\n",
    "                         dtype={'stationID': np.string0, 'accumPrec': np.int32}, parse_dates=[[1, 2, 3]])\n",
    "    atmos = pd.concat([temp, prec], axis=1)\n",
    "    for i in range(len(ATMOS)): #2\n",
    "        for j in range(1, len(ATMOS[i])): #48\n",
    "            if i == 0:\n",
    "                tempe = pd.read_table(TEMPS[j], sep='\\s+', usecols=[0, 4, 5, 6, 8, 9],\n",
    "                                        names=['stationID', 'year', 'month', 'day', 'maxDayTemp', 'minDayTemp'],\n",
    "                                        dtype={'stationID': np.string0, 'maxDayTemp': np.float32,\n",
    "                                               'minDayTemp': np.float32}, parse_dates=[[1, 2, 3]])\n",
    "                temp = pd.concat([temp, tempe])\n",
    "            if i == 1:\n",
    "                prece = pd.read_table(PRECS[j], sep='\\s+', usecols=[0, 4, 5, 6, 9],\n",
    "                                     names=['stationID', 'year', 'month', 'day', 'accumPrec'],\n",
    "                                     dtype={'stationID': np.string0, 'accumPrec': np.int32}, parse_dates=[[1, 2, 3]])\n",
    "                prec = pd.concat([prec, prece])\n",
    "    atmose = pd.concat([temp, prec], axis=1)\n",
    "    atmos = pd.concat([atmos, atmose])\n",
    "    atmos = _post_processing(atmos)\n",
    "    return atmos\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    atmos = combine_all_files()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
